package com.cynen.p42u.task;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import java.io.*;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
 * 爬虫的入口,主要目的是将爬虫的配置类,拆分单独配置.
 */
@Component
public class JavJob {

    @Value("${jav.init.url}")
    private  String initUrl;

    @Autowired
    private JavInfoPipline javInfoPipline;

    // 启动任务的定时任务.
    @Scheduled(initialDelay = 100*1000,fixedDelay = 300*1000)
    public void start_spider(){
        //读取文件.
        InputStream stream = JavJob.class.getClassLoader().getResourceAsStream("config.properties");
        Properties prop = new Properties();
        try {
            prop.load(stream);
        } catch (IOException e) {
            e.printStackTrace();
        }
        // 为了获取外部配置的Cookie值.
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(prop.getProperty("cookie_file_path")));
            String readLine = bufferedReader.readLine();
            prop.setProperty("Cookie",readLine); // 不配置成键值对,直接存cookie值.
            bufferedReader.close();
        }catch (Exception e){
            System.out.println("cookie_file_path 配置文件异常");
        }

        // 构建Map
        Map<String,String> info = new HashMap<>();
        info.put("charset",prop.getProperty("charset"));
        info.put("userAgent",prop.getProperty("userAgent"));
        info.put("Cookie",prop.getProperty("Cookie"));
        info.put("Domain",prop.getProperty("Domain"));

        System.out.println("爬虫抓取开始.....");
        Spider.create(new PicScrawp(info))
                .addUrl(initUrl) // 初始的url
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(1000000)))
                .thread(10)
                .addPipeline(javInfoPipline)
                .run();
    }
}
